import tensorflow as tf
tf.__version__

'2.2.0'

The Keras functional API

Coding tutorials

1. Multiple inputs and outputs

2. Tensors and Variables

3. Accessing model layers

4. Freezing layers

---

Multiple inputs and outputs

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline

Load the acute inflammations dataset

The acute inflammations was created by a medical expert as a data set to test the expert system, which will perform the presumptive diagnosis of two diseases of the urinary system. You can find out more about the dataset here.

Attribute information:

Inputs:

• Temperature of patient : 35C-42C
• Occurrence of nausea : yes/no
• Lumbar pain : yes/no
• Urine pushing (continuous need for urination) : yes/no
• Micturition pains : yes/no
• Burning of urethra, itch, swelling of urethra outlet : yes/no

Outputs:

• decision 1: Inflammation of urinary bladder : yes/no
• decision 2: Nephritis of renal pelvis origin : yes/no

Load the data

# Load the dataset
from sklearn.model_selection import train_test_split
pd_dat = pd.read_csv('data/diagnosis.csv')
dataset = pd_dat.values

# Build train and test data splits
X_train, X_test, Y_train, Y_test = train_test_split(dataset[:,:6], dataset[:,6:], test_size=0.33)

# Assign training and testing inputs/outputs
temp_train, nocc_train, lumbp_train, up_train, mict_train, bis_train = np.transpose(X_train)
temp_test, nocc_test, lumbp_test, up_test, mict_test, bis_test = np.transpose(X_test)
inflam_train, nephr_train = Y_train[:, 0], Y_train[:, 1]
inflam_test, nephr_test = Y_test[:, 0], Y_test[:, 1]

Build the model

# Build the input layers
from tensorflow.keras import Input, layers
shape_inputs = (1,)
temperature = Input(shape = shape_inputs, name = 'temp')
nausea_occurence = Input(shape = shape_inputs, name = 'nocc')
lumbar_pain = Input(shape = shape_inputs, name = 'lumbp')
urine_pushing = Input(shape = shape_inputs, name = 'up')
micturition_pains = Input(shape = shape_inputs, name = 'mict')
bis = Input(shape = shape_inputs, name = 'bis')

# Create a list of all the inputs
list_inputs = [temperature, nausea_occurence, lumbar_pain, urine_pushing, 
               micturition_pains, bis]

# Merge all input features into a single large vector
x = layers.concatenate(list_inputs)

# Use a logistic regression classifier for disease prediction
inflammation_pred = layers.Dense(1, activation = 'sigmoid', name = 'inflam')(x)
nephritis_pred = layers.Dense(1, activation = 'sigmoid', name = 'nephr')(x)

# Create a list of all the outputs
list_outputs = [inflammation_pred, nephritis_pred]

# Create the model object
model = tf.keras.Model(inputs = list_inputs, outputs = list_outputs)

Plot the model

# Display the multiple input/output model
tf.keras.utils.plot_model(model, 'multi_input_output_model.png', show_shapes = True)

Compile the model

# Compile the model
model.compile (
    optimizer = tf.keras.optimizers.RMSprop(1e-3),
    loss = ['binary_crossentropy', 'binary_crossentropy'],
    metrics = {'inflam': ['acc'], 'nephr': ['acc']},
    loss_weights = [1., 0.2]
)

Fit the model

# Define training inputs and outputs
inputs_train = {
    'temp': temp_train, 'nocc': nocc_train, 'lumbp': lumbp_train,
    'up': up_train, 'mict': mict_train, 'bis': bis_train
}
outputs_train = {'inflam': inflam_train, 'nephr': nephr_train}

# Train the model
history = model.fit(inputs_train, outputs_train, epochs = 1000, batch_size = 128, verbose = 0)

Plot the learning curves

# Plot the training accuracy

acc_keys = [k for k in history.history.keys() if k in ('inflam_acc', 'nephr_acc')] 
loss_keys = [k for k in history.history.keys() if not k in acc_keys]

for k, v in history.history.items():
    if k in acc_keys:
        plt.figure(1)
        plt.plot(v)
    else:
        plt.figure(2)
        plt.plot(v)

plt.figure(1)
plt.title('Accuracy vs. epochs')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(acc_keys, loc='upper right')

plt.figure(2)
plt.title('Loss vs. epochs')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(loss_keys, loc='upper right')

plt.show()

# Evaluate the model
model.evaluate (
    [temp_test, nocc_test, lumbp_test, up_test, mict_test, bis_test],
    [inflam_test, nephr_test],
    verbose = 2
)

2/2 - 0s - loss: 0.4909 - inflam_loss: 0.3909 - nephr_loss: 0.5000 - inflam_acc: 1.0000 - nephr_acc: 0.9250

[0.4909321367740631,
 0.39093565940856934,
 0.49998220801353455,
 1.0,
 0.925000011920929]

---

Tensors and Variables

import numpy as np

Create Variable objects

# Create Variable objects of different type with tf.Variable

strings = tf.Variable(['Hello world!'], tf.string)
floats  = tf.Variable([3.14159, 2.71828], tf.float64)
ints = tf.Variable([1, 2, 3], tf.int32)
complexs = tf.Variable([25.9 - 7.39j, 1.23 - 4.91j], tf.complex128)

# Initialise a Variable value
tf.Variable(tf.constant(4.2, shape = (3, 3)))

<tf.Variable 'Variable:0' shape=(3, 3) dtype=float32, numpy=
array([[4.2, 4.2, 4.2],
       [4.2, 4.2, 4.2],
       [4.2, 4.2, 4.2]], dtype=float32)>

Use and modify Variable values

# Use the value of a Variable
v = tf.Variable(0.0)
w = v + 1  # w is a tf.Tensor which is computed based on the value of v.
print(type(w))

<class 'tensorflow.python.framework.ops.EagerTensor'>

# Increment the value of a Variable
v.assign_add(1)
v

<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=1.0>

# Decrement the value of a Variable
v.assign_sub(1)
v

<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>

Create Tensor objects

Create a constant tensor and print its type as well as its shape:

# Create a constant Tensor
x = tf.constant([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
print(x)
print('dtype:', x.dtype)
print('shape:', x.shape)

tf.Tensor(
[[1 2 3]
 [4 5 6]
 [7 8 9]], shape=(3, 3), dtype=int32)
dtype: <dtype: 'int32'>
shape: (3, 3)

# Obtain the value as a numpy array
x.numpy()

array([[1, 2, 3],
       [4, 5, 6],
       [7, 8, 9]], dtype=int32)

# Create a Tensor of type float32
x = tf.constant([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype = tf.float32)
x.dtype

tf.float32

# Create coefficients
coeffs = np.arange(16)

# Initialise shapes
shape1 = [8, 2]
shape2 = [4, 4]
shape3 = [2, 2, 2, 2]

# Create Tensors of different shape
a = tf.constant(coeffs, shape=shape1)
print("\n a:\n ", a)
b = tf.constant(coeffs, shape=shape2)
print("\n b:\n ", b)
c = tf.constant(coeffs, shape=shape3)
print("\n c:\n ", c)

 a:
  tf.Tensor(
[[ 0  1]
 [ 2  3]
 [ 4  5]
 [ 6  7]
 [ 8  9]
 [10 11]
 [12 13]
 [14 15]], shape=(8, 2), dtype=int64)

 b:
  tf.Tensor(
[[ 0  1  2  3]
 [ 4  5  6  7]
 [ 8  9 10 11]
 [12 13 14 15]], shape=(4, 4), dtype=int64)

 c:
  tf.Tensor(
[[[[ 0  1]
   [ 2  3]]

  [[ 4  5]
   [ 6  7]]]


 [[[ 8  9]
   [10 11]]

  [[12 13]
   [14 15]]]], shape=(2, 2, 2, 2), dtype=int64)

Useful Tensor operations

# Create a constant Tensor
t = tf.constant(np.arange(80), shape=[5, 2, 8])

# Get the rank of a Tensor
rank = tf.rank(t)

# Display the rank
print("rank: ", rank)

rank:  tf.Tensor(3, shape=(), dtype=int32)

# Reshape a Tensor
t2 = tf.reshape(t, [8, 10])

# Display the new shape
print("t2.shape: ", t2.shape)

t2.shape:  (8, 10)

# Create ones, zeros, identity and constant Tensors
ones = tf.ones(shape = (2, 3))
zeros = tf.zeros(shape = (2, 4))
eye = tf.eye(3)
tensor7 = tf.constant(7.0, shape = [2, 2])

# Display the created tensors

print("\n Ones:\n ", ones)
print("\n Zeros:\n ", zeros)
print("\n Identity:\n ", eye)
print("\n Tensor filled with 7: ", tensor7)

 Ones:
  tf.Tensor(
[[1. 1. 1.]
 [1. 1. 1.]], shape=(2, 3), dtype=float32)

 Zeros:
  tf.Tensor(
[[0. 0. 0. 0.]
 [0. 0. 0. 0.]], shape=(2, 4), dtype=float32)

 Identity:
  tf.Tensor(
[[1. 0. 0.]
 [0. 1. 0.]
 [0. 0. 1.]], shape=(3, 3), dtype=float32)

 Tensor filled with 7:  tf.Tensor(
[[7. 7.]
 [7. 7.]], shape=(2, 2), dtype=float32)

# Create a ones Tensor and a zeros Tensor
t1 = tf.ones(shape = (2, 2))
t2 = tf.zeros(shape = (2, 2))

# Concatentate two Tensors
concat0 = tf.concat([t1, t2], 0)
concat1 = tf.concat([t1, t2], 1)

# Display the concatenated tensors
print(concat0)
print(concat1)

tf.Tensor(
[[1. 1.]
 [1. 1.]
 [0. 0.]
 [0. 0.]], shape=(4, 2), dtype=float32)
tf.Tensor(
[[1. 1. 0. 0.]
 [1. 1. 0. 0.]], shape=(2, 4), dtype=float32)

# Create a constant Tensor
t = tf.constant(np.arange(24), shape = (3, 2, 4))
print("\n t shape: ", t.shape)

 t shape:  (3, 2, 4)

# Expanding the rank of Tensors
t1 = tf.expand_dims(t, 0)
t2 = tf.expand_dims(t, 1)
t3 = tf.expand_dims(t, 3)

# Display the shapes after tf.expand_dims
print (
    "\n After expanding dims:\n t1 shape: ", t1.shape,
    "\n t2 shape: ", t2.shape,
    "\n t3 shape: ", t3.shape
)

 After expanding dims:
 t1 shape:  (1, 3, 2, 4) 
 t2 shape:  (3, 1, 2, 4) 
 t3 shape:  (3, 2, 4, 1)

# Squeezing redundant dimensions
t1 = tf.squeeze(t1, 0)
t2 = tf.squeeze(t2, 1)
t3 = tf.squeeze(t3, 3)

# Display the shapes after tf.squeeze
print (
    "\n After squeezing:\n t1 shape: ", t1.shape,
    "\n t2 shape: ", t2.shape,
    "\n t3 shape: ", t3.shape
)

 After squeezing:
 t1 shape:  (3, 2, 4) 
 t2 shape:  (3, 2, 4) 
 t3 shape:  (3, 2, 4)

# Slicing a Tensor
x = tf.constant([1, 2, 3, 4, 5, 6, 7])
print(x[1:4])

tf.Tensor([2 3 4], shape=(3,), dtype=int32)

Doing maths with Tensors

# Create two constant Tensors
c = tf.constant([[1.0, 2.0], [3.0, 4.0]])
d = tf.constant([[1.0, 1.0], [0.0, 1.0]])

# Matrix multiplication
matmul_cd = tf.matmul(c, d)

# Display the result
print("\n tf.matmul(c,d):\n", matmul_cd)

 tf.matmul(c,d):
 tf.Tensor(
[[1. 3.]
 [3. 7.]], shape=(2, 2), dtype=float32)

# Elementwise operations
c_times_d = c * d
c_plus_d = c + d
c_minus_d = c - d
c_div_c = c / c

# Display the results
print("\n c*d:\n", c_times_d)
print("\n c+d:\n", c_plus_d)
print("\n c-d:\n", c_minus_d)
print("\n c/c:\n", c_div_c)

 c*d:
 tf.Tensor(
[[1. 2.]
 [0. 4.]], shape=(2, 2), dtype=float32)

 c+d:
 tf.Tensor(
[[2. 3.]
 [3. 5.]], shape=(2, 2), dtype=float32)

 c-d:
 tf.Tensor(
[[0. 1.]
 [3. 3.]], shape=(2, 2), dtype=float32)

 c/c:
 tf.Tensor(
[[1. 1.]
 [1. 1.]], shape=(2, 2), dtype=float32)

# Create Tensors
a = tf.constant([[2, 3], [3, 3]])
b = tf.constant([[8, 7], [2, 3]])
x = tf.constant([[-6.89 + 1.78j], [-2.54 + 2.15j]])

# Absolute value of a Tensor
absx = tf.abs(x)

# Power of a Tensor
powaa = tf.pow(a, a)

# Display the results
print("\n ", absx)
print("\n ", powaa)

  tf.Tensor(
[[7.11621388]
 [3.32777704]], shape=(2, 1), dtype=float64)

  tf.Tensor(
[[ 4 27]
 [27 27]], shape=(2, 2), dtype=int32)

Randomly sampled constant tensors

# Create a Tensor with samples from a Normal distribution
tn = tf.random.normal(shape = (2, 2), mean = 0, stddev = 1.)
tn

<tf.Tensor: shape=(2, 2), dtype=float32, numpy=
array([[ 1.1122508 ,  0.02933991],
       [ 0.45304555, -2.482614  ]], dtype=float32)>

# Create a Tensor with samples from a Uniform distribution
tu = tf.random.uniform(shape = (2, 1), minval = 0, maxval = 10, dtype = 'int32')
tu

<tf.Tensor: shape=(2, 1), dtype=int32, numpy=
array([[9],
       [7]], dtype=int32)>

# Create a Tensor with samples from a Poisson distribution
tu = tf.random.poisson((2, 2), 5)
tu

<tf.Tensor: shape=(2, 2), dtype=float32, numpy=
array([[4., 4.],
       [4., 6.]], dtype=float32)>

# More maths operations
d = tf.square(tn)
e = tf.exp(d)
f = tf.cos(c)
print(d)
print(e)
print(f)

tf.Tensor(
[[1.2371019e+00 8.6083042e-04]
 [2.0525026e-01 6.1633725e+00]], shape=(2, 2), dtype=float32)
tf.Tensor(
[[  3.4456134   1.0008612]
 [  1.2278323 475.0274   ]], shape=(2, 2), dtype=float32)
tf.Tensor(
[[ 0.5403023 -0.4161468]
 [-0.9899925 -0.6536436]], shape=(2, 2), dtype=float32)

---

Accessing model layers

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

Load the pre-trained model

In this section, we aim to demonstrate accessing layer attributes within a model.

Let's get started by loading the VGG19 pre-trained model from the keras.applications library, which is a very deep network trained on more than a million images from the ImageNet database. The network is trained to classify images into 1000 object categories.

# Load the VGG19 model

from tensorflow.keras.applications import VGG19
# vgg_model = VGG19()
vgg_model = VGG19(weights = None)
vgg_model.load_weights('models/vgg19_weights_tf_dim_ordering_tf_kernels.h5')

# Get the inputs, layers and display the summary
vgg_input = vgg_model.input
vgg_layers = vgg_model.layers
vgg_model.summary()

Model: "vgg19"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         [(None, 224, 224, 3)]     0         
_________________________________________________________________
block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         
_________________________________________________________________
block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     
_________________________________________________________________
block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         
_________________________________________________________________
block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    
_________________________________________________________________
block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    
_________________________________________________________________
block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    
_________________________________________________________________
block3_conv4 (Conv2D)        (None, 56, 56, 256)       590080    
_________________________________________________________________
block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         
_________________________________________________________________
block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   
_________________________________________________________________
block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   
_________________________________________________________________
block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   
_________________________________________________________________
block4_conv4 (Conv2D)        (None, 28, 28, 512)       2359808   
_________________________________________________________________
block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         
_________________________________________________________________
block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   
_________________________________________________________________
block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   
_________________________________________________________________
block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   
_________________________________________________________________
block5_conv4 (Conv2D)        (None, 14, 14, 512)       2359808   
_________________________________________________________________
block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         
_________________________________________________________________
flatten (Flatten)            (None, 25088)             0         
_________________________________________________________________
fc1 (Dense)                  (None, 4096)              102764544 
_________________________________________________________________
fc2 (Dense)                  (None, 4096)              16781312  
_________________________________________________________________
predictions (Dense)          (None, 1000)              4097000   
=================================================================
Total params: 143,667,240
Trainable params: 143,667,240
Non-trainable params: 0
_________________________________________________________________

Build a model to access the layer outputs

from tensorflow.keras.models import Model

# Build a model that returns the layer outputs
layer_outputs = [layer.output for layer in vgg_layers]
features = Model(inputs = vgg_input, outputs = layer_outputs)

# Plot the model
tf.keras.utils.plot_model(features, 'vgg19_model.png', show_shapes = True)

# Test the model on a random input
img = np.random.random((1, 224, 224, 3)).astype('float32')
extracted_features = features(img)

Load the 'cool cat' picture

In Zambia’s South Luangwa National Park, a photographer had been watching a pride of lions while they slept off a feast from a buffalo kill. When this female walked away, he anticipated that she might be going for a drink and so he positioned his vehicle on the opposite side of the waterhole. The cool cat picture is one of the highly commended 2018 Image from Wildlife Photographer of the Year.

# Display the original image
import IPython.display as display
from PIL import Image
display.display(Image.open('data/cool_cat.jpg'))

Visualise network features from the input image

# Preprocess the image

from tensorflow.keras.applications.vgg19 import preprocess_input
from tensorflow.keras.preprocessing import image

img_path = 'data/cool_cat.jpg'
img = image.load_img(img_path, target_size = (224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis = 0)
x = preprocess_input(x)

# Extract the features
extracted_features = features(x)

# Visualise the input channels
f1 = extracted_features[0]
print('\n f1.shape: ', f1.shape)

imgs = f1[0, :, :]
plt.figure(figsize = (15, 15))
for n in range(3):
    ax = plt.subplot(1, 3, n + 1)
    plt.imshow(imgs[:, :, n])
    plt.axis('off')
plt.subplots_adjust(wspace = 0.01, hspace = 0.01)

 f1.shape:  (1, 224, 224, 3)

# Visualise some features in the first hidden layer
f2 = extracted_features[1]
print('\n f2.shape: ', f2.shape)

imgs = f2[0, :, :]
plt.figure(figsize = (15, 15))
for n in range(16):
    ax = plt.subplot(4, 4, n + 1)
    plt.imshow(imgs[:, :, n])
    plt.axis('off')
plt.subplots_adjust(wspace = 0.01, hspace = 0.01)

 f2.shape:  (1, 224, 224, 64)

# Build a model to extract features by layer name
extracted_features_block1_pool = \
    Model(inputs = features.input, outputs = features.get_layer('block1_pool').output)
block1_pool_features = extracted_features_block1_pool.predict(x)

# Visualise some features from the extracted layer output
imgs = block1_pool_features[0, :, :]
plt.figure(figsize = (15, 15))
for n in range(16):
    ax = plt.subplot(4, 4, n + 1)
    plt.imshow(imgs[:, :, n])
    plt.axis('off')
plt.subplots_adjust(wspace = 0.01, hspace = 0.01)

# Extract features from a layer deeper in the network
extracted_features_block5_conv4 = \
    Model(inputs = features.input, outputs = features.get_layer('block5_conv4').output)
block5_conv4_features = extracted_features_block5_conv4.predict(x)

# Visualise some features from the extracted layer output
imgs = block5_conv4_features[0, :, :]
plt.figure(figsize = (15, 15))
for n in range(16):
    ax = plt.subplot(4, 4, n + 1)
    plt.imshow(imgs[:, :, n])
    plt.axis('off')
plt.subplots_adjust(wspace = 0.01, hspace = 0.01)

---

Freezing layers

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

Build the model

# Build a small Sequential model
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
model = Sequential ([
    layers.Dense (
        4, input_shape = (4,), activation = 'relu',
        kernel_initializer = 'random_uniform', bias_initializer = 'ones'
    ),
    layers.Dense(2, activation = 'relu', kernel_initializer = 'lecun_normal', bias_initializer = 'ones'),
    layers.Dense(4, activation = 'softmax'),
])

# Display the model summary
model.summary()

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                (None, 4)                 20        
_________________________________________________________________
dense_1 (Dense)              (None, 2)                 10        
_________________________________________________________________
dense_2 (Dense)              (None, 4)                 12        
=================================================================
Total params: 42
Trainable params: 42
Non-trainable params: 0
_________________________________________________________________

Examine the weight matrix variation over training

def get_weights(model):
    return [e.weights[0].numpy() for e in model.layers]

def get_biases(model):
    return [e.bias.numpy() for e in model.layers]

def plot_delta_weights(W0_layers, W1_layers, b0_layers, b1_layers):
    plt.figure(figsize = (8, 8))
    for n in range(3):
        delta_l = W1_layers[n] - W0_layers[n]
        print('Layer ' + str(n) + ': bias variation: ', np.linalg.norm(b1_layers[n] - b0_layers[n]))
        ax = plt.subplot(1, 3, n + 1)
        plt.imshow(delta_l)
        plt.title('Layer ' + str(n))
        plt.axis('off')
    plt.colorbar()
    plt.suptitle('Weight matrices variation')

# Retrieve the weights and biases
W0_layers = get_weights(model)
b0_layers = get_biases(model)

# Construct a synthetic dataset
x_train = np.random.random((100, 4))
y_train = x_train
x_test = np.random.random((20, 4))
y_test = x_test

# Compile and fit the model
model.compile(optimizer = 'adam', loss = 'mse', metrics = ['acc'])
model.fit(x_train, y_train, epochs = 50, verbose = False);

# Retrieve weights and biases
W1_layers = get_weights(model)
b1_layers = get_biases(model)

# Plot the variation
plot_delta_weights(W0_layers, W1_layers, b0_layers, b1_layers)

Layer 0: bias variation:  0.028452259
Layer 1: bias variation:  0.014293373
Layer 2: bias variation:  0.092317775

Freeze layers at build time

# Count the trainable and non trainable variables before the freezing
n_trainable_variables = len(model.trainable_variables)
n_non_trainable_variables = len(model.non_trainable_variables)

# Display the number of trainable and non trainable variables before the freezing
print (
    "\n Before freezing:\n\t Number of trainable variables: ", n_trainable_variables,
    "\n\t Number of non trainable variables: ", n_non_trainable_variables
)

 Before freezing:
	 Number of trainable variables:  6 
	 Number of non trainable variables:  0

# Build the model
model = Sequential ([
    layers.Dense (
        4, input_shape = (4,), activation = 'relu',
        kernel_initializer = 'random_uniform', bias_initializer = 'ones', trainable = False
    ),
    layers.Dense(2, activation = 'relu', kernel_initializer = 'lecun_normal', bias_initializer = 'ones'),
    layers.Dense(4, activation = 'softmax'),
])

# Count the trainable and non trainable variables after the freezing
n_trainable_variables = len(model.trainable_variables)
n_non_trainable_variables = len(model.non_trainable_variables)

# Display the number of trainable and non trainable variables after the freezing
print (
    "\n After freezing:\n\t Number of trainable variables: ", n_trainable_variables,
    "\n\t Number of non trainable variables: ", n_non_trainable_variables
)

 After freezing:
	 Number of trainable variables:  4 
	 Number of non trainable variables:  2

# Retrieve weights and biases
W0_layers = get_weights(model)
b0_layers = get_biases(model)

# Compile and fit the model
model.compile(optimizer = 'adam', loss = 'mse', metrics = ['acc'])
model.fit(x_train, y_train, epochs = 50, verbose = False);

# Retrieve weights and biases
W1_layers = get_weights(model)
b1_layers = get_biases(model)

# Plot the variation
plot_delta_weights(W0_layers, W1_layers, b0_layers, b1_layers)

Layer 0: bias variation:  0.0
Layer 1: bias variation:  0.1509809
Layer 2: bias variation:  0.33442858

Freeze layers of a pre-built model

# Count the trainable and non trainable variables before the freezing
print (
    "\n Before freezing:\n\t Number of trainable variables: ", len(model.trainable_variables),
    "\n\t Number of non trainable variables: ", len(model.non_trainable_variables)
)

 Before freezing:
	 Number of trainable variables:  4 
	 Number of non trainable variables:  2

# Freeze the second layer
model.layers[1].trainable = False

# Count the trainable and non trainable variables after the freezing
print (
    "\n After freezing:\n\t Number of trainable variables: ", len(model.trainable_variables),
    "\n\t Number of non trainable variables: ", len(model.non_trainable_variables)
)

 After freezing:
	 Number of trainable variables:  2 
	 Number of non trainable variables:  4

# Compile and fit the model
model.compile(optimizer = 'adam', loss = 'mse', metrics = ['acc'])
model.fit(x_train, y_train, epochs = 50, verbose = False);

# Retrieve weights and biases
W2_layers = get_weights(model)
b2_layers = get_biases(model)

# Plot the variation
plot_delta_weights(W1_layers, W2_layers, b1_layers, b2_layers)

Layer 0: bias variation:  0.0
Layer 1: bias variation:  0.0
Layer 2: bias variation:  0.27730897